{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn import datasets\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, cross_val_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, SimpleRNN, LSTM, GRU, Dropout, Activation, Bidirectional, Conv2D, MaxPooling2D\n",
    "from keras import optimizers\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.wrappers.scikit_learn import KerasClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load dataset\n",
    "- digits dataset in scikit-learn\n",
    "- url: http://scikit-learn.org/stable/auto_examples/datasets/plot_digits_last_image.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = datasets.load_digits()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACxNJREFUeJzt3fuLXPUZx/HPp5vErRqTYqxKNjShaEAqNZqmhIjQBEus\nokJL3YCWSmGhoCiGihZL239A0h+KIFErmBpsVBDrBVsVK6QxF1M1txKDJRvURLwHTLLm6Q87gShp\n92zmnO+ZeXy/YHEvw36fQd45Z2ZnztcRIQA5fa3tAQA0h8CBxAgcSIzAgcQIHEiMwIHECBxIjMCB\nxAgcSGxKE790mk+JQZ3WxK9u1dissvfpnHPeL7bWvoMzi601OHqk2FpxZKzYWiV9poM6HIc80e0a\nCXxQp+n7XtbEr27Vez9eXHS9X61cW2yt32y+ptha59/2drG1xt55t9haJW2Iv1e6HafoQGIEDiRG\n4EBiBA4kRuBAYgQOJEbgQGIEDiRWKXDby23vsr3b9h1NDwWgHhMGbntA0h8lXSHpAkkrbF/Q9GAA\nulflCL5I0u6I2BMRhyWtlVTudY0ATlqVwGdL2nvc16Od7wHocbW92cT2iKQRSRrUqXX9WgBdqHIE\n3ydpznFfD3W+9wURcW9ELIyIhVN1Sl3zAehClcA3SjrP9jzb0yQNS3qi2bEA1GHCU/SIGLN9k6Rn\nJQ1Iuj8itjU+GYCuVXoMHhFPSXqq4VkA1IxXsgGJETiQGIEDiRE4kBiBA4kROJAYgQOJETiQWCM7\nm2RVcqcRSRqe/kGxtVbN/LTYWn/d8myxtS753S+LrSVJs+5dX3S9iXAEBxIjcCAxAgcSI3AgMQIH\nEiNwIDECBxIjcCAxAgcSq7Kzyf2299t+o8RAAOpT5Qj+J0nLG54DQAMmDDwiXpL0foFZANSMx+BA\nYmxdBCRW2xGcrYuA3sMpOpBYlT+TPSxpvaT5tkdt/6L5sQDUocreZCtKDAKgfpyiA4kROJAYgQOJ\nETiQGIEDiRE4kBiBA4kROJBY329dNLb0kmJrDU/fWmwtSbpi+XCxtWa8trPYWj99eVmxtd5f8Hmx\ntSRpVtHVJsYRHEiMwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIjMCBxKpcdHGO7Rdsb7e9zfYt\nJQYD0L0qr0Ufk7QyIrbYni5ps+3nImJ7w7MB6FKVvcnejogtnc8/kbRD0uymBwPQvUm9m8z2XEkL\nJG04wc/YugjoMZWfZLN9uqRHJd0aER9/+edsXQT0nkqB256q8bjXRMRjzY4EoC5VnkW3pPsk7YiI\nu5sfCUBdqhzBl0i6QdJS21s7Hz9qeC4ANaiyN9nLklxgFgA145VsQGIEDiRG4EBiBA4kRuBAYgQO\nJEbgQGIEDiTW93uTfXZmubtw1/4Li60lSUcL7hdW0sbXv932CF8ZHMGBxAgcSIzAgcQIHEiMwIHE\nCBxIjMCBxAgcSIzAgcSqXHRx0PYrtv/V2bro9yUGA9C9Kq/zPCRpaUR82rl88su2n46IfzY8G4Au\nVbnoYkj6tPPl1M5HNDkUgHpU3fhgwPZWSfslPRcRJ9y6yPYm25uO6FDdcwI4CZUCj4jPI+IiSUOS\nFtn+zgluw9ZFQI+Z1LPoEfGhpBckLW9mHAB1qvIs+lm2Z3Y+/7qkyyXlfKMykEyVZ9HPlfSg7QGN\n/4PwSEQ82exYAOpQ5Vn01zS+JziAPsMr2YDECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIrP+3LvpG\nuX+j1qxfXGwtSTpfrxRdr5QpMw4XW2vso2nF1upFHMGBxAgcSIzAgcQIHEiMwIHECBxIjMCBxAgc\nSIzAgcQqB965NvqrtrkeG9AnJnMEv0XSjqYGAVC/qjubDEm6UtLqZscBUKeqR/BVkm6XdLTBWQDU\nrMrGB1dJ2h8Rmye4HXuTAT2myhF8iaSrbb8laa2kpbYf+vKN2JsM6D0TBh4Rd0bEUETMlTQs6fmI\nuL7xyQB0jb+DA4lN6oouEfGipBcbmQRA7TiCA4kROJAYgQOJETiQGIEDiRE4kBiBA4kROJBY329d\nNPhBuTe4fe/CN4utJUkfFVxryjlnF1vrugv+7/uWavXI05cWW6sXcQQHEiNwIDECBxIjcCAxAgcS\nI3AgMQIHEiNwIDECBxKr9Eq2zhVVP5H0uaSxiFjY5FAA6jGZl6r+ICLea2wSALXjFB1IrGrgIelv\ntjfbHmlyIAD1qXqKfmlE7LP9TUnP2d4ZES8df4NO+COSNKhTax4TwMmodASPiH2d/+6X9LikRSe4\nDVsXAT2myuaDp9mefuxzST+U9EbTgwHoXpVT9LMlPW772O3/HBHPNDoVgFpMGHhE7JH03QKzAKgZ\nfyYDEiNwIDECBxIjcCAxAgcSI3AgMQIHEiNwILG+37rojF3lNvj57dCTxdaSpJ+N3FZsranXHii2\nVknz7lzf9git4ggOJEbgQGIEDiRG4EBiBA4kRuBAYgQOJEbgQGIEDiRWKXDbM22vs73T9g7bi5se\nDED3qr5U9Q+SnomIn9ieJnHhc6AfTBi47RmSLpP0c0mKiMOSDjc7FoA6VDlFnyfpgKQHbL9qe3Xn\n+ugAelyVwKdIuljSPRGxQNJBSXd8+Ua2R2xvsr3piA7VPCaAk1El8FFJoxGxofP1Oo0H/wVsXQT0\nngkDj4h3JO21Pb/zrWWStjc6FYBaVH0W/WZJazrPoO+RdGNzIwGoS6XAI2KrpIUNzwKgZrySDUiM\nwIHECBxIjMCBxAgcSIzAgcQIHEiMwIHECBxIrO/3Jjv62s5ia113z8pia0nSXSsfLrbWqjeXFVtr\n40UDxdb6quMIDiRG4EBiBA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4kNmHgtufb3nrcx8e2by0xHIDu\nTPhS1YjYJekiSbI9IGmfpMcbngtADSZ7ir5M0psR8Z8mhgFQr8m+2WRY0gnfAWF7RNKIJA2y+SjQ\nEyofwTubHlwt6S8n+jlbFwG9ZzKn6FdI2hIR7zY1DIB6TSbwFfofp+cAelOlwDv7gV8u6bFmxwFQ\np6p7kx2UdGbDswCoGa9kAxIjcCAxAgcSI3AgMQIHEiNwIDECBxIjcCAxR0T9v9Q+IGmybymdJem9\n2ofpDVnvG/erPd+KiLMmulEjgZ8M25siYmHbczQh633jfvU+TtGBxAgcSKyXAr+37QEalPW+cb96\nXM88BgdQv146ggOoWU8Ebnu57V22d9u+o+156mB7ju0XbG+3vc32LW3PVCfbA7Zftf1k27PUyfZM\n2+ts77S9w/bitmfqRuun6J1rrf9b41eMGZW0UdKKiNje6mBdsn2upHMjYovt6ZI2S7q23+/XMbZv\nk7RQ0hkRcVXb89TF9oOS/hERqzsXGj01Ij5se66T1QtH8EWSdkfEnog4LGmtpGtanqlrEfF2RGzp\nfP6JpB2SZrc7VT1sD0m6UtLqtmepk+0Zki6TdJ8kRcThfo5b6o3AZ0vae9zXo0oSwjG250paIGlD\nu5PUZpWk2yUdbXuQms2TdEDSA52HH6s71yPsW70QeGq2T5f0qKRbI+Ljtufplu2rJO2PiM1tz9KA\nKZIulnRPRCyQdFBSXz8n1AuB75M057ivhzrf63u2p2o87jURkeWKtEskXW37LY0/nFpq+6F2R6rN\nqKTRiDh2prVO48H3rV4IfKOk82zP6zypMSzpiZZn6ppta/yx3I6IuLvteeoSEXdGxFBEzNX4/6vn\nI+L6lseqRUS8I2mv7fmdby2T1NdPik52b7LaRcSY7ZskPStpQNL9EbGt5bHqsETSDZJet721871f\nR8RTLc6Eid0saU3nYLNH0o0tz9OV1v9MBqA5vXCKDqAhBA4kRuBAYgQOJEbgQGIEDiRG4EBiBA4k\n9l+8Q5/pEyhkXAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x237f470d390>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "label:  0\n"
     ]
    }
   ],
   "source": [
    "plt.imshow(data.images[0])    # show first number in the dataset\n",
    "plt.show()\n",
    "print('label: ', data.target[0])    # label = '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_data = data.images\n",
    "y_data = data.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1797, 8, 8)\n",
      "(1797,)\n"
     ]
    }
   ],
   "source": [
    "# shape of data\n",
    "print(X_data.shape)    # (8 X 8) format\n",
    "print(y_data.shape)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_data, y_data, test_size = 0.2, random_state = 777)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1437, 8, 8)\n",
      "(360, 8, 8)\n",
      "(1437,)\n",
      "(360,)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Simple RNN model\n",
    "- Simple RNN model with only one vanilla RNN layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def simple_rnn_model():\n",
    "    adam = optimizers.Adam(lr = 0.01)\n",
    "    model = Sequential()\n",
    "    model.add(SimpleRNN(10, input_shape = (8,8)))\n",
    "    model.add(Activation('softmax'))\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    \n",
    "    return model  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "simple_rnn_model = KerasClassifier(build_fn = simple_rnn_model, epochs = 100, batch_size = 25, verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(n_splits=10, shuffle = True, random_state = 777)    # 10-fold cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cross validate the results\n",
    "results = cross_val_score(simple_rnn_model, X_train, y_train, cv = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.512370629298\n",
      "0.0877594107786\n"
     ]
    }
   ],
   "source": [
    "# print out mean and standard deviation of accuracy results\n",
    "print(results.mean())\n",
    "print(results.std())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Stacked RNN model\n",
    "- Two-layer vanilla RNN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def stacked_rnn_model():\n",
    "    adam = optimizers.Adam(lr = 0.01)\n",
    "    model = Sequential()\n",
    "    model.add(SimpleRNN(10, return_sequences = True, input_shape = (8,8)))\n",
    "    model.add(SimpleRNN(10))\n",
    "    model.add(Activation('softmax'))\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    \n",
    "    return model  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stacked_rnn_model = KerasClassifier(build_fn = stacked_rnn_model, epochs = 100, batch_size = 25, verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(n_splits=10, shuffle = True, random_state = 777)    # 10-fold cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cross validate the results\n",
    "results = cross_val_score(stacked_rnn_model, X_train, y_train, cv = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.782629487682\n",
      "0.0443044186784\n"
     ]
    }
   ],
   "source": [
    "# print out mean and standard deviation of accuracy results\n",
    "print(results.mean())\n",
    "print(results.std())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Bidirectional RNN model\n",
    "- RNN model with one bi-directional layer\n",
    "- As there are actually two hidden layers in a bi-directional layer, the number of units of a RNN neuron should be halved"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bidirectional_rnn_model():\n",
    "    adam = optimizers.Adam(lr = 0.01)\n",
    "    model = Sequential()\n",
    "    model.add(Bidirectional(SimpleRNN(5, return_sequences = False), input_shape = (8,8)))    # number of output units is halved\n",
    "    model.add(Activation('softmax'))\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    \n",
    "    return model  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bidirectional_rnn_model = KerasClassifier(build_fn = bidirectional_rnn_model, epochs = 100, batch_size = 25, verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cross validate the results\n",
    "results = cross_val_score(bidirectional_rnn_model, X_train, y_train, cv = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.455588528752\n",
      "0.105806469411\n"
     ]
    }
   ],
   "source": [
    "# print out mean and standard deviation of accuracy results\n",
    "print(results.mean())\n",
    "print(results.std())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Simple LSTM model\n",
    "- LSTM model with one layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def simple_lstm_model():\n",
    "    adam = optimizers.Adam(lr = 0.01)\n",
    "    model = Sequential()\n",
    "    model.add(LSTM(10, return_sequences = False, input_shape = (8,8)))\n",
    "    model.add(Activation('softmax'))\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    \n",
    "    return model  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "simple_lstm_model = KerasClassifier(build_fn = simple_lstm_model, epochs = 100, batch_size = 25, verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cross validate the results\n",
    "results = cross_val_score(simple_lstm_model, X_train, y_train, cv = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.735386777075\n",
      "0.0757424070027\n"
     ]
    }
   ],
   "source": [
    "# print out mean and standard deviation of accuracy results\n",
    "print(results.mean())\n",
    "print(results.std())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Stacked LSTM model\n",
    "- LSTM model with two layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def stacked_lstm_model():\n",
    "    adam = optimizers.Adam(lr = 0.01)\n",
    "    model = Sequential()\n",
    "    model.add(LSTM(10, return_sequences = True, input_shape = (8,8)))\n",
    "    model.add(LSTM(10))\n",
    "    model.add(Activation('softmax'))\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    \n",
    "    return model  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stacked_lstm_model = KerasClassifier(build_fn = stacked_lstm_model, epochs = 100, batch_size = 25, verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cross validate the results\n",
    "results = cross_val_score(stacked_lstm_model, X_train, y_train, cv = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.915160591419\n",
      "0.0325069346011\n"
     ]
    }
   ],
   "source": [
    "# print out mean and standard deviation of accuracy results\n",
    "print(results.mean())\n",
    "print(results.std())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. Bidirectional LSTM model\n",
    "- LSTM model with one bi-directional layer\n",
    "- As there are actually two hidden layers in a bi-directional layer, the number of units of a LSTM neuron should be halved"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bidirectional_lstm_model():\n",
    "    adam = optimizers.Adam(lr = 0.01)\n",
    "    model = Sequential()\n",
    "    model.add(Bidirectional(LSTM(5, return_sequences = False), input_shape = (8,8)))    # number of output units is halved\n",
    "    model.add(Activation('softmax'))\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    \n",
    "    return model  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bidirectional_lstm_model = KerasClassifier(build_fn = bidirectional_lstm_model, epochs = 100, batch_size = 25, verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cross validate the results\n",
    "results = cross_val_score(bidirectional_lstm_model, X_train, y_train, cv = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.561767569655\n",
      "0.115548047249\n"
     ]
    }
   ],
   "source": [
    "# print out mean and standard deviation of accuracy results\n",
    "print(results.mean())\n",
    "print(results.std())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. Simple GRU model\n",
    "- GRU model with one layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def simple_gru_model():\n",
    "    adam = optimizers.Adam(lr = 0.01)\n",
    "    model = Sequential()\n",
    "    model.add(GRU(10, return_sequences = False, input_shape = (8,8)))\n",
    "    model.add(Activation('softmax'))\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    \n",
    "    return model  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "simple_gru_model = KerasClassifier(build_fn = simple_gru_model, epochs = 100, batch_size = 25, verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cross validate the results\n",
    "results = cross_val_score(simple_gru_model, X_train, y_train, cv = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.754681983122\n",
      "0.0646756393836\n"
     ]
    }
   ],
   "source": [
    "# print out mean and standard deviation of accuracy results\n",
    "print(results.mean())\n",
    "print(results.std())                          "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. Stacked GRU model\n",
    "- GRU model with two GRU layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def stacked_gru_model():\n",
    "    adam = optimizers.Adam(lr = 0.01)\n",
    "    model = Sequential()\n",
    "    model.add(GRU(10, return_sequences = True, input_shape = (8,8)))\n",
    "    model.add(GRU(10))\n",
    "    model.add(Activation('softmax'))\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    \n",
    "    return model  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stacked_gru_model = KerasClassifier(build_fn = stacked_gru_model, epochs = 100, batch_size = 25, verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cross validate the results\n",
    "results = cross_val_score(stacked_gru_model, X_train, y_train, cv = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.86831371306\n",
      "0.0647763236999\n"
     ]
    }
   ],
   "source": [
    "# print out mean and standard deviation of accuracy results\n",
    "print(results.mean())\n",
    "print(results.std())     "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9. Bidirectional GRU model\n",
    "- GRU model with one bi-directional layer\n",
    "- As there are actually two hidden layers in a bi-directional layer, the number of units of a GRU neuron should be halved"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bidirectional_gru_model():\n",
    "    adam = optimizers.Adam(lr = 0.01)\n",
    "    model = Sequential()\n",
    "    model.add(Bidirectional(GRU(5, return_sequences = False), input_shape = (8,8)))    # number of output units is halved\n",
    "    model.add(Activation('softmax'))\n",
    "    model.compile(loss = 'categorical_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    \n",
    "    return model  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bidirectional_gru_model = KerasClassifier(build_fn = bidirectional_gru_model, epochs = 100, batch_size = 25, verbose = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cross validate the results\n",
    "results = cross_val_score(bidirectional_gru_model, X_train, y_train, cv = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.585051039768\n",
      "0.082875157127\n"
     ]
    }
   ],
   "source": [
    "# print out mean and standard deviation of accuracy results\n",
    "print(results.mean())\n",
    "print(results.std())     "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
